# -*- coding: utf-8 -*-
# @Time    : 2024/1/23 20:37
# @Author  : micah
# @File    : 1.asyncio结合requests完成数据提取.py
# @Software: PyCharm


import asyncio
import requests
from functools import partial
from bs4 import BeautifulSoup

url = 'https://movie.douban.com/top250?start={}&filter='
headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
}

loop = asyncio.get_event_loop()

async def get_movie_info(page):
    # 发送请求
    response = await loop.run_in_executor(None, partial(requests.get, url.format(page), headers=headers))
    soup = BeautifulSoup(response.text, 'lxml')
    div_list = soup.find_all('div', class_='hd')
    for title in div_list:
        print(title.text)


tasks = [loop.create_task(get_movie_info(page * 25)) for page in range(10)]
loop.run_until_complete(asyncio.wait(tasks))
